<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2016-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Running Pipelines</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="CDAP 数据流管道" href="index.html" />
    <link rel="next" title="Plugin Management" href="plugin-management.html" />
    <link rel="prev" title="Creating Pipelines" href="creating-pipelines.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: user-guide -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><b><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview.html"> 概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/campaign.html">MySQL 客户数据</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/nytimes-xml.html">纽约时报 XML 数据推送</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/stocks.html">股票选择</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/fitbit.html">物联网 IoT 设备数据</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-preparation/index.html"> 数据预处理</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/concepts.html">      概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/directives/index.html">      数据处理指令</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/catalog-lookup.html">catalog-lookup</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/change-column-case.html">change-column-case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/changing-case.html">changing-case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/cleanse-column-names.html">cleanse-column-names</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/columns-replace.html">columns-replace</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/copy.html">copy</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/cut-character.html">cut-character</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/decode.html">decode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/diff-date.html">diff-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/drop.html">drop</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/encode.html">encode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/extract-regex-groups.html">extract-regex-groups</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/fail.html">fail</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/fill-null-or-empty.html">fill-null-or-empty</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/filter-row-if-matched.html">filter-row-if-matched</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/filter-row-if-true.html">filter-row-if-true</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/filter-rows-on.html">filter-rows-on</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/find-and-replace.html">find-and-replace</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/flatten.html">flatten</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/format-date.html">format-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/format-unix-timestamp.html">format-unix-timestamp</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/generate-uuid.html">generate-uuid</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/hash.html">hash</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/increment-variable.html">increment-variable</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/index-split.html">index-split</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/invoke-http.html">invoke-http</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/json-path.html">json-path</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/keep.html">keep</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/mask-number.html">mask-number</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/mask-shuffle.html">mask-shuffle</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/merge.html">merge</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-avro-file.html">parse-as-avro-file</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-avro.html">parse-as-avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-csv.html">parse-as-csv</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-date.html">parse-as-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-excel.html">parse-as-excel</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-fixed-length.html">parse-as-fixed-length</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-hl7.html">parse-as-hl7</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-json.html">parse-as-json</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-log.html">parse-as-log</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-simple-date.html">parse-as-simple-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-xml.html">parse-as-xml</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-timestamp.html">parse-timestamp</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-xml-to-json.html">parse-xml-to-json</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/quantize.html">quantize</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/rename.html">rename</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/send-to-error.html">send-to-error</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-charset.html">set-charset</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-column.html">set-column</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-columns.html">set-columns</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-record-delim.html">set-record-delim</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-type.html">set-type</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-variable.html">set-variable</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-by-separator.html">split-by-separator</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-email.html">split-email</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-to-columns.html">split-to-columns</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-to-rows.html">split-to-rows</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-url.html">split-url</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/stemming.html">stemming</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/swap.html">swap</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/table-lookup.html">table-lookup</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/text-distance.html">text-distance</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/text-metric.html">text-metric</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/trim.html">trim</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/url-decode.html">url-decode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/url-encode.html">url-encode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/write-as-csv.html">write-as-csv</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/write-as-json-map.html">write-as-json-map</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/write-as-json-object.html">write-as-json-object</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/xpath.html">xpath</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/functions/index.html">      函数</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/json-functions.html">JSON 函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/type-functions.html">类型函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/geofence-functions.html">地理围栏函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/dq-functions.html">数据质量函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/date-functions.html">日期函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/ddl-functions.html">DDL 函数</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/service/index.html">      服务</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/admin.html">行政和管理服务</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/connection-properties.html">连接属性</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/connections.html">连接服务</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/execution.html">数据处理指令执行</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/request.html">请求格式规范</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/schema-registry.html">Schema 注册库</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/services.html">数据预处理服务</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/performance.html">性能</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/exclusion-and-aliasing.html">排除与别名</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 数据流管道</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="concepts-design.html"> 概念与设计</a></li>
<li class="toctree-l2"><a class="reference internal" href="getting-started.html"> 入门指南</a></li>
<li class="toctree-l2"><a class="reference internal" href="studio.html"> CDAP 数据流设计器</a></li>
<li class="toctree-l2"><a class="reference internal" href="creating-pipelines.html"> 创建数据流管道</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> 运行数据流管道</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugin-management.html"> 插件管理</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugins/index.html"> 插件参考</a><ul>
<li class="toctree-l3"><a class="reference internal" href="plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../mmds/index.html"> 数据分析</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../mmds/concepts.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mmds/feature-gen.html"> Feature Generation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mmds/modeling.html"> Modeling</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mmds/example.html"> Example</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="running-pipelines">
<span id="user-guide-cdap-pipelines-running-pipelines"></span><h1>Running Pipelines<a class="headerlink" href="#running-pipelines" title="Permalink to this headline">🔗</a></h1>
<p>Pipeline can be started, stopped, and controlled using:</p>
<ul class="simple">
<li><a class="reference internal" href="#cdap-pipelines-running-pipelines-within-cdap-pipelines-ui"><span class="std std-ref">CDAP Pipelines UI</span></a></li>
<li><span class="xref std std-ref">CDAP UI</span></li>
<li><span class="xref std std-ref">CDAP CLI</span></li>
<li>Command line tools, using the <span class="xref std std-ref">Lifecycle HTTP RESTful API</span></li>
</ul>
<div class="section" id="running-a-pipeline-within-the-cdap-pipelines-ui">
<span id="cdap-pipelines-running-pipelines-within-cdap-pipelines-ui"></span><h2>Running a Pipeline within the CDAP Pipelines UI<a class="headerlink" href="#running-a-pipeline-within-the-cdap-pipelines-ui" title="Permalink to this headline">🔗</a></h2>
<p>From within the CDAP Pipelines UI, you can start and stop pipelines.</p>
<p>For a <strong>batch pipeline,</strong> you can start or stop (“suspend”) its schedule. It will then begin
at the next scheduled time. (To change the schedule requires creating a new pipeline with
a new schedule.)</p>
<p>For a <strong>real-time pipeline,</strong> you simply start or stop the pipeline.</p>
<p>You can set the arguments used to start the pipeline (both as preferences and runtime
arguments) and view details of a pipeline, such as its configuration, settings, stages,
schemas, and logs.</p>
</div>
<div class="section" id="runtime-arguments">
<h2>Runtime Arguments<a class="headerlink" href="#runtime-arguments" title="Permalink to this headline">🔗</a></h2>
<p>You may want to create a pipeline that has several configuration settings that are not
known at pipeline creation time, but that are set at the start of the each pipeline run.</p>
<p>For instance, you might want a pipeline that reads from a database (a source) and writes
to a table (a sink). The name of the database source and name of the table sink might
change from run to run and you need to specify those values as input before starting a
run.</p>
<p>You might want to create a pipeline with a particular action at the start of the run.
The action could, based on some logic, provide the name of the database to use as a source
and the name of the table to write as a sink. The next stage in the pipeline might use
this information to read and write from appropriate sources and sinks.</p>
<p>To do this, CDAP supports the use of macros that will, at runtime, will be evaluated
and substituted for. The macros support recursive (nested) expansion and use a simple
syntax. These macros are <a class="reference internal" href="creating-pipelines.html#cdap-pipelines-runtime-arguments-macros"><span class="std std-ref">defined in the pipeline configuration</span></a> and in the runtime arguments.</p>
<p>Runtime arguments are resolved by sourcing them from the application preferences, the
runtime arguments, and the workflow token. Precedence is with the workflow token having
the highest precedence.</p>
<p>These arguments and preferences can be set with the CDAP UI, the CDAP CLI or the
<span class="xref std std-ref">Program Lifecycle</span> and <span class="xref std std-ref">Preferences</span> HTTP RESTful APIs.</p>
<p>Details of usage and examples are explained in the section on <a class="reference internal" href="creating-pipelines.html#cdap-pipelines-runtime-arguments-macros"><span class="std std-ref">runtime arguments and
macros</span></a>.</p>
</div>
<div class="section" id="re-running-a-pipeline">
<h2>Re-running a Pipeline<a class="headerlink" href="#re-running-a-pipeline" title="Permalink to this headline">🔗</a></h2>
<p>When a pipeline is re-run, any previously-set runtime arguments will have been discarded
and will need to be set again, if required. Any previously-set application preferences will
be retained and re-used.</p>
</div>
<div class="section" id="notifications">
<h2>Notifications<a class="headerlink" href="#notifications" title="Permalink to this headline">🔗</a></h2>
<p>When a pipeline is completed, notifications can be sent by using one or more <em>post-run
actions</em>. These are <a class="reference internal" href="creating-pipelines.html#cdap-pipelines-creating-pipelines-post-run-actions"><span class="std std-ref">set in the pipeline configuration</span></a>.</p>
<p>These <a class="reference internal" href="plugins/post-run-plugins/index.html#cdap-pipelines-plugins-post-run-plugins"><span class="std std-ref">post-run plugins</span></a> allow for emails,
database queries, and a general HTTP callback action.</p>
</div>
<div class="section" id="logs">
<h2>Logs<a class="headerlink" href="#logs" title="Permalink to this headline">🔗</a></h2>
<p>As pipelines run, they create entries in the CDAP logs.</p>
<p>Logs for a pipeline can be obtained using the same tools as any other CDAP application and
program, and are described in the <span class="xref std std-ref">Administration manual, Logging and Monitoring</span> and the <span class="xref std std-ref">Logging HTTP RESTful API</span>.</p>
<p>Script transform steps can write to logs, as described in the section in developing
plugins on <span class="xref std std-ref">script transformations</span>.</p>
</div>
<div class="section" id="metrics">
<h2>Metrics<a class="headerlink" href="#metrics" title="Permalink to this headline">🔗</a></h2>
<p>As pipelines run, they create both system and user metrics.</p>
<p>System metrics for a pipeline can be obtained using the same tools as any other CDAP
application and program, and are described in the <span class="xref std std-ref">Administration manual, metrics</span> and the <span class="xref std std-ref">Metrics HTTP RESTful API</span>.</p>
<p>Script transform steps can create metrics, as described in the section in developing
plugins on <span class="xref std std-ref">script transformations</span>.</p>
<p>For instance, if you have a real-time pipeline named “demoPipeline” with three stages
(<em>DataGenerator</em>, <em>JavaScript</em>, and <em>Table</em>), then you can discover the available metrics
using a <cite>curl</cite> command, such as (reformatted for display):</p>
<blockquote>
<div>
<script type="text/javascript">

  $(function tabbedparsedliteral45() {
    var tabs = ['linux', 'windows'];
    var mapping = {'windows': 'windows', 'linux': 'linux'};
    var tabSetID = 'linux-windows';
    for (var i = 0; i < tabs.length; i++) {
      var tab = tabs[i];
      $("#tabbedparsedliteral45 .example-tab-" + tab).click(changeExampleTab(tab, mapping, "tabbedparsedliteral45", tabSetID));
    }
  });

</script>
<div id="tabbedparsedliteral45" class="tabbed-parsed-literal dependent-linux-windows">
<ul class="tabbed-parsed-literal nav-tabs">
<li class="example-tab example-tab-linux active"><a href="#">Linux</a></li>
<li class="example-tab example-tab-windows "><a href="#">Windows</a></li>
</ul>

<div class="tab-contents">

<div class="tab-pane tab-pane-linux active">
<div class="code code-tab">
<div class="highlight-console">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s2">&quot;\n&quot;</span> -X POST <span class="s2">&quot;http://localhost:11015/v3/metrics/search?target=metric&amp;tag=namespace:default&amp;tag=app:demoPipeline&quot;</span>

<span class="go">[&quot;system.app.log.debug&quot;,&quot;system.app.log.info&quot;,&quot;system.app.log.warn&quot;,&quot;system.dataset.</span>
<span class="go">store.bytes&quot;,&quot;system.dataset.store.ops&quot;,&quot;system.dataset.store.reads&quot;,&quot;system.dataset.</span>
<span class="go">store.writes&quot;,&quot;system.metrics.emitted.count&quot;,&quot;user.DataGenerator.records.out&quot;,&quot;user.</span>
<span class="go">JavaScript.record.count&quot;,&quot;user.JavaScript.records.in&quot;,&quot;user.JavaScript.records.out&quot;,&quot;</span>
<span class="go">user.Table.records.in&quot;,&quot;user.Table.records.out&quot;,&quot;user.metrics.emitted.count&quot;]</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
<div class="tab-pane tab-pane-windows ">
<div class="code code-tab">
<div class="highlight-shell-session">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">&gt;</span> curl -X POST <span class="s2">&quot;http://localhost:11015/v3/metrics/search?target=metric&amp;tag=namespace:default&amp;tag=app:demoPipeline&quot;</span>

<span class="go">[&quot;system.app.log.debug&quot;,&quot;system.app.log.info&quot;,&quot;system.app.log.warn&quot;,&quot;system.dataset.</span>
<span class="go">store.bytes&quot;,&quot;system.dataset.store.ops&quot;,&quot;system.dataset.store.reads&quot;,&quot;system.dataset.</span>
<span class="go">store.writes&quot;,&quot;system.metrics.emitted.count&quot;,&quot;user.DataGenerator.records.out&quot;,&quot;user.</span>
<span class="go">JavaScript.record.count&quot;,&quot;user.JavaScript.records.in&quot;,&quot;user.JavaScript.records.out&quot;,&quot;</span>
<span class="go">user.Table.records.in&quot;,&quot;user.Table.records.out&quot;,&quot;user.metrics.emitted.count&quot;]</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
</div>
</div>
</div></blockquote>
<p>In this case, the user metric <em>“user.JavaScript.record.count”</em> was incremented in the JavaScript stage using:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span>context.getMetrics().count(&#39;record.count&#39;, 1);
</pre></div>
</div>
<p>The value of the metric can be retrieved with:</p>
<blockquote>
<div>
<script type="text/javascript">

  $(function tabbedparsedliteral46() {
    var tabs = ['linux', 'windows'];
    var mapping = {'windows': 'windows', 'linux': 'linux'};
    var tabSetID = 'linux-windows';
    for (var i = 0; i < tabs.length; i++) {
      var tab = tabs[i];
      $("#tabbedparsedliteral46 .example-tab-" + tab).click(changeExampleTab(tab, mapping, "tabbedparsedliteral46", tabSetID));
    }
  });

</script>
<div id="tabbedparsedliteral46" class="tabbed-parsed-literal dependent-linux-windows">
<ul class="tabbed-parsed-literal nav-tabs">
<li class="example-tab example-tab-linux active"><a href="#">Linux</a></li>
<li class="example-tab example-tab-windows "><a href="#">Windows</a></li>
</ul>

<div class="tab-contents">

<div class="tab-pane tab-pane-linux active">
<div class="code code-tab">
<div class="highlight-console">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s2">&quot;\n&quot;</span> -X POST <span class="s2">&quot;localhost:11015/v3/metrics/query?tag=namespace:default&amp;tag=app:etlRealtime6&amp;metric=user.JavaScript.record.count&amp;aggregate=true&quot;</span>

<span class="go">{&quot;startTime&quot;:0,&quot;endTime&quot;:1468884338,&quot;series&quot;:[{&quot;metricName&quot;:&quot;user.JavaScript.record.</span>
<span class="go">count&quot;,&quot;grouping&quot;:{},&quot;data&quot;:[{&quot;time&quot;:0,&quot;value&quot;:170}]}],&quot;resolution&quot;:&quot;2147483647s&quot;}</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
<div class="tab-pane tab-pane-windows ">
<div class="code code-tab">
<div class="highlight-shell-session">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">&gt;</span> curl -X POST <span class="s2">&quot;localhost:11015/v3/metrics/query?tag=namespace:default&amp;tag=app:etlRealtime6&amp;metric=user.JavaScript.record.count&amp;aggregate=true&quot;</span>

<span class="go">{&quot;startTime&quot;:0,&quot;endTime&quot;:1468884338,&quot;series&quot;:[{&quot;metricName&quot;:&quot;user.JavaScript.record.</span>
<span class="go">count&quot;,&quot;grouping&quot;:{},&quot;data&quot;:[{&quot;time&quot;:0,&quot;value&quot;:170}]}],&quot;resolution&quot;:&quot;2147483647s&quot;}</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
</div>
</div>
</div></blockquote>
<p>Using the CDAP CLI, you can retrieve the value with:</p>
<blockquote>
<div>
<script type="text/javascript">

  function change_tabbedparsedliteral47_ExampleTab(tab) {
    return function(e) {
      e.preventDefault();
      var scrollOffset = $(this).offset().top - $(document).scrollTop();
      $("#tabbedparsedliteral47 .tab-pane").removeClass("active");
      $("#tabbedparsedliteral47 .tab-pane-" + tab).addClass("active");
      $("#tabbedparsedliteral47 .example-tab").removeClass("active");
      $("#tabbedparsedliteral47 .example-tab-" + tab).addClass("active");
      $(document).scrollTop($(this).offset().top - scrollOffset);
    }
  }

  $(function() {
    var tabs = ['cdap-cli'];
    for (var i = 0; i < tabs.length; i++) {
      var tab = tabs[i];
      $("#tabbedparsedliteral47 .example-tab-" + tab).click(change_tabbedparsedliteral47_ExampleTab(tab));
    }
  });

</script>
<div id="tabbedparsedliteral47" class="tabbed-parsed-literal independent">
<ul class="tabbed-parsed-literal nav-tabs">
<li class="example-tab example-tab-cdap-cli active"><a href="#">CDAP CLI</a></li>
</ul>

<div class="tab-contents">

<div class="tab-pane tab-pane-cdap-cli active">
<div class="code code-tab">
<div class="highlight-shell-session">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="go">cdap &gt; get metric value user.JavaScript.record.count &#39;app=demoPipeline&#39;</span>

<span class="go">Start time: 0</span>
<span class="go">End time: 1468884640</span>

<span class="go">Series: user.JavaScript.record.count</span>
<span class="go">+===================+</span>
<span class="go">| timestamp | value |</span>
<span class="go">+===================+</span>
<span class="go">| 0         | 170   |</span>
<span class="go">+===================+</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
</div>
</div>
</div></blockquote>
</div>
<div class="section" id="error-record-handling">
<span id="cdap-pipelines-running-pipelines-error-record-handling"></span><h2>Error Record Handling<a class="headerlink" href="#error-record-handling" title="Permalink to this headline">🔗</a></h2>
<p>To handle the problem of validating records and handling any subsequent errors, certain
transform stages are available that can check that a record matches specified criteria.
Records that fail the criteria can be discarded, and appropriate messages written to a
configurable error dataset.</p>
<p>These transform plugins support error record handling:</p>
<ul class="simple">
<li><a class="reference external" href="plugins/transforms/javascript-transform.html">JavaScript</a></li>
<li><a class="reference external" href="plugins/transforms/pythonevaluator-transform.html">Python Evaluator</a></li>
<li><a class="reference external" href="plugins/transforms/validator-transform.html">Validator</a></li>
<li><a class="reference external" href="plugins/transforms/xmlparser-transform.html">XML Parser</a></li>
</ul>
<p>See the <a class="reference internal" href="plugins/shared-plugins/core.html#cdap-pipelines-plugins-shared-core-validator"><span class="std std-ref">Core Validator</span></a> for examples
and additional information.</p>
</div>
<div class="section" id="configuring-resources">
<h2>Configuring Resources<a class="headerlink" href="#configuring-resources" title="Permalink to this headline">🔗</a></h2>
<p>Resources for pipelines can be configured as any other CDAP workflow application.</p>
<p>See the 管理手册 sections on <span class="xref std std-ref">Scaling Instances</span> and
<span class="xref std std-ref">Resource Guarantees for CDAP Programs in YARN</span>.</p>
</div>
<div class="section" id="upgrading-a-pipeline">
<span id="cdap-pipelines-operating-upgrading-pipeline"></span><h2>Upgrading a Pipeline<a class="headerlink" href="#upgrading-a-pipeline" title="Permalink to this headline">🔗</a></h2>
<p>If you wish to upgrade 4.2.x pipelines, you can use the ETL upgrade
tool packaged with the <strong>distributed</strong> version of CDAP. You would want to run this tool to
upgrade applications that were created with earlier versions of the artifacts, that you
would like to open in the 6.1.1 version of CDAP Studio.</p>
<p>The tool will connect to an instance of CDAP, look for any applications that use
4.2.x versions of the system artifacts, and then update the
application to use the 6.1.1 version of those artifacts.</p>
<p>CDAP must be running when you run the command:</p>
<div class="highlight container">
<pre class="literal-block">
<span class="gp">$</span> java -cp /opt/cdap/master/libexec/cdap-etl-tools-6.1.1.jar io.cdap.cdap.etl.tool.UpgradeTool -u http://&lt;host&gt;:&lt;port&gt; -e /tmp/failedUpgrades upgrade
</pre>
</div>
<p>The first argument is the host and port for the <span class="xref std std-ref">CDAP router</span>. The second argument is a directory to write the
configurations of any pipelines that could not be upgraded. A pipeline may fail to upgrade
if the new version of a plugin used in the pipeline is not backwards compatible. For
example, this may happen if the plugin added a new required property.</p>
<p>You can also upgrade just the ETL applications within a specific namespace:</p>
<div class="highlight container">
<pre class="literal-block">
<span class="gp">$</span> java -cp /opt/cdap/master/libexec/cdap-etl-tools-6.1.1.jar io.cdap.cdap.etl.tool.UpgradeTool -u http://&lt;host&gt;:&lt;port&gt; -n &lt;namespace-id&gt; upgrade
</pre>
</div>
<p>You can also upgrade just one ETL application:</p>
<div class="highlight container">
<pre class="literal-block">
<span class="gp">$</span> java -cp /opt/cdap/master/libexec/cdap-etl-tools-6.1.1.jar io.cdap.cdap.etl.tool.UpgradeTool -u http://&lt;host&gt;:&lt;port&gt; -n &lt;namespace-id&gt; -p &lt;app-name&gt; upgrade
</pre>
</div>
<p>If you have authentication turned on, you also need to store an access token in a file and pass the file to the tool:</p>
<div class="highlight container">
<pre class="literal-block">
<span class="gp">$</span> java -cp /opt/cdap/master/libexec/cdap-etl-tools-6.1.1.jar io.cdap.cdap.etl.tool.UpgradeTool -u http://&lt;host&gt;:&lt;port&gt; -a &lt;tokenfile&gt; upgrade
</pre>
</div>
<p>For instance, if you have obtained an access token (as shown in the example in the
<cite>security documentation &lt;testing-security&gt;</cite>) such as:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="p">{</span><span class="s">&quot;access_token&quot;</span><span class="p">:</span><span class="s">&quot;AghjZGFwAI7e8p65Uo7OpfG5UrD87psGQE0u0sFDoqxtacdRR5GxEb6bkTypP7mXdqvqqnLmfxOS&quot;</span><span class="p">,</span>
  <span class="s">&quot;token_type&quot;</span><span class="p">:</span><span class="s">&quot;Bearer&quot;</span><span class="p">,</span><span class="s">&quot;expires_in&quot;</span><span class="p">:</span><span class="mi">86400</span><span class="p">}</span>
</pre></div>
</div>
<p>The access token itself (<code class="docutils literal notranslate"><span class="pre">AghjZGFwAI7e8p65Uo7OpfG5UrD87psGQE0u0sFDoqxtacdRR5GxEb6bkTypP7mXdqvqqnLmfxOS</span></code>)
would be placed in a file and then the file’s path would be used in the above command.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Creating Pipelines" href="creating-pipelines.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Plugin Management" href="plugin-management.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>